{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!byzerllm deploy --pretrained_model_type custom/auto \\\n",
    "--infer_backend vllm \\\n",
    "--model_path /home/winubuntu/models/Qwen2-7B \\\n",
    "--cpus_per_worker 0.001 \\\n",
    "--gpus_per_worker 1 \\\n",
    "--num_workers 1 \\\n",
    "--infer_params backend.max_model_len=28000 backend.enable_lora=true backend.max_lora_rank=2 \\\n",
    "--model qwen_1_8b_chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import byzerllm\n",
    "\n",
    "llm = byzerllm.ByzerLLM()\n",
    "llm.setup_default_model_name(\"qwen_1_8b_chat\")\n",
    "llm.chat_oai(conversations=[{\n",
    "    \"role\": \"user\",\n",
    "    \"content\": \"Hello, how are you?\"\n",
    "}],llm_config={\"adapter_name_or_path\": \"/home/winubuntu/projects/jupyter-workspace/微调/qwen_lora\",\"lora_name\":\"default\",\"lora_int_id\":1})"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
